{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AI第五周作业——基于模型的协同过滤"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math as mt\n",
    "from scipy.sparse.linalg import * #used for matrix multiplication\n",
    "from scipy.sparse.linalg import svds\n",
    "from scipy.sparse import csc_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>song</th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
       "      <td>SOADQPP12A67020C82</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
       "      <td>SOAFTRR12AF72A8D4D</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
       "      <td>SOANQFY12AB0183239</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
       "      <td>SOAYATB12A6701FD50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d6589314c0a9bcbca4fee0c93b14bc402363afea</td>\n",
       "      <td>SOBOAFP12A8C131F36</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       user                song  play_count\n",
       "0  d6589314c0a9bcbca4fee0c93b14bc402363afea  SOADQPP12A67020C82           5\n",
       "1  d6589314c0a9bcbca4fee0c93b14bc402363afea  SOAFTRR12AF72A8D4D           1\n",
       "2  d6589314c0a9bcbca4fee0c93b14bc402363afea  SOANQFY12AB0183239           1\n",
       "3  d6589314c0a9bcbca4fee0c93b14bc402363afea  SOAYATB12A6701FD50           1\n",
       "4  d6589314c0a9bcbca4fee0c93b14bc402363afea  SOBOAFP12A8C131F36           5"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读入数据\n",
    "df_music = pd.read_csv('triplet_dataset_sub_5000_song.csv') \n",
    "df_music.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>play_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3.712888e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2.369724e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.619127e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>4.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>5.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         play_count\n",
       "count  3.712888e+06\n",
       "mean   2.369724e+00\n",
       "std    1.619127e+00\n",
       "min    1.000000e+00\n",
       "25%    1.000000e+00\n",
       "50%    2.000000e+00\n",
       "75%    4.000000e+00\n",
       "max    5.000000e+00"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_music.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 将表格转换成稀疏矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(99612, 4001)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.sparse import coo_matrix\n",
    "\n",
    "small_set = df_music\n",
    "user_codes = small_set.user.drop_duplicates().reset_index()\n",
    "song_codes = small_set.song.drop_duplicates().reset_index()\n",
    "user_codes.rename(columns={'index':'user_index'}, inplace=True)\n",
    "song_codes.rename(columns={'index':'song_index'}, inplace=True)\n",
    "song_codes['song_index_value'] = list(song_codes.index)\n",
    "user_codes['user_index_value'] = list(user_codes.index)\n",
    "small_set = pd.merge(small_set,song_codes,how='left')\n",
    "small_set = pd.merge(small_set,user_codes,how='left')\n",
    "mat_candidate = small_set[['user_index_value','song_index_value','play_count']]\n",
    "\n",
    "data_array = mat_candidate.play_count.values\n",
    "row_array = mat_candidate.user_index_value.values\n",
    "col_array = mat_candidate.song_index_value.values\n",
    "\n",
    "data_sparse = coo_matrix((data_array, (row_array, col_array)),dtype=float)\n",
    "data_sparse.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[5. 1. 1. ... 0. 0. 0.]\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [3. 3. 3. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "data_sparse_arr = data_sparse.toarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 找出打分最多前5首歌曲"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "\n",
    "def findMostRateSong(arr, songNum, commonNum):\n",
    "\n",
    "    songRate = [ 0 for i in range(songNum)]\n",
    "    \n",
    "    for i in range(len(arr)):\n",
    "        #for j in range(len(arr[0])):\n",
    "        for j in range(songNum):\n",
    "            if arr[i][j] > 0.1:\n",
    "                songRate[j] += 1\n",
    "    \n",
    "    songRateSort= copy.copy(songRate)\n",
    "    songRateSort.sort(reverse = True)                 \n",
    "    #print(songRate)\n",
    "    #print(songRateSort)\n",
    "                       \n",
    "    MostRateSongCount = [songRateSort[i] for i in range(commonNum)]\n",
    "                       \n",
    "    MostRateSongID = []\n",
    "   \n",
    "    for i in range(len(songRate)):\n",
    "        for j in range(len(MostRateSongCount)):\n",
    "            if songRate[i] == MostRateSongCount[j]:\n",
    "                MostRateSongID.append(i)           \n",
    "                   \n",
    "    return MostRateSongID, MostRateSongCount    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def findMostSongUserID(arr, MostRateSongID):\n",
    "    \n",
    "    UserRateID = []\n",
    "    SongRate = []\n",
    "    \n",
    "    for i in range(len(arr)):\n",
    "        isRate = True\n",
    "        uSongRate = []\n",
    "        for j in range(len(MostRateSongID)):\n",
    "            if arr[i][j] < 0.1:\n",
    "                isRate = False\n",
    "            else:\n",
    "                uSongRate.append(arr[i][j])\n",
    "        if isRate:\n",
    "            SongRate.append(uSongRate)\n",
    "            UserRateID.append(i)\n",
    "            \n",
    "    return UserRateID, SongRate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def UserFilter(UserASongRate, UserBSongRate):\n",
    "    \n",
    "    UserARateSum = 0\n",
    "    for i in range(len(UserASongRate) - 1):\n",
    "        UserARateSum += UserASongRate[i]    \n",
    "    UserAAverage = UserARateSum / (len(UserASongRate) - 1)\n",
    "    #print('UserAAverage:', UserAAverage)\n",
    "    \n",
    "    UserBRateSum = 0\n",
    "    for i in range(len(UserBSongRate) - 1):\n",
    "        UserBRateSum += UserBSongRate[i]\n",
    "    UserBAverage = UserBRateSum / (len(UserBSongRate) - 1)\n",
    "    #print('UserBAverage:', UserBAverage)\n",
    "    \n",
    "    sum = 0\n",
    "    for i in range(len(UserASongRate) - 1):\n",
    "        sum += (UserASongRate[i] - UserAAverage) * (UserBSongRate[i] - UserBAverage)\n",
    "    numerator = sum\n",
    "    #print('numerator:', numerator)\n",
    "    \n",
    "    sum = 0\n",
    "    for i in range(len(UserASongRate) - 1):\n",
    "        sum += pow((UserASongRate[i] - UserAAverage), 2)\n",
    "    leftPart = np.sqrt(sum)    \n",
    "    \n",
    "    sum = 0\n",
    "    for i in range(len(UserBSongRate) - 1):\n",
    "        sum += pow((UserBSongRate[i] - UserBAverage), 2)\n",
    "    rightPart = np.sqrt(sum)\n",
    "    \n",
    "    denominator = leftPart * rightPart\n",
    "    #print('denominator:', denominator)\n",
    "    \n",
    "    sim = numerator / denominator\n",
    "    \n",
    "    return sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MostRateSongID: [29, 680, 707, 907, 1044]\n",
      "MostRateSongCount: [18626, 17635, 15138, 14945, 14687]\n"
     ]
    }
   ],
   "source": [
    "MostRateSongID , MostRateSongCount = findMostRateSong(data_sparse_arr, len(data_sparse_arr[0]), 5)\n",
    "print('MostRateSongID:', MostRateSongID)\n",
    "print('MostRateSongCount:', MostRateSongCount)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UserRateID: [0, 19443, 22222, 30288, 31637, 69824, 76596, 80576, 88020]\n",
      "SongRate: [[5.0, 1.0, 1.0, 1.0, 5.0], [1.0, 1.0, 1.0, 4.0, 2.0], [1.0, 5.0, 1.0, 1.0, 2.0], [1.0, 3.0, 1.0, 1.0, 1.0], [1.0, 1.0, 2.0, 1.0, 1.0], [1.0, 1.0, 3.0, 1.0, 3.0], [1.0, 2.0, 5.0, 5.0, 4.0], [5.0, 1.0, 2.0, 1.0, 1.0], [2.0, 5.0, 5.0, 5.0, 1.0]]\n"
     ]
    }
   ],
   "source": [
    "UserRateID, SongRate = findMostSongUserID(data_sparse_arr, MostRateSongID)\n",
    "print('UserRateID:',UserRateID)\n",
    "print('SongRate:',SongRate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "SongRate[0][4] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_svd(urm, K):\n",
    "    U, s, Vt = svds(urm, K)\n",
    "\n",
    "    dim = (len(s), len(s))\n",
    "    S = np.zeros(dim, dtype=np.float32)\n",
    "    for i in range(0, len(s)):\n",
    "        S[i,i] = mt.sqrt(s[i])\n",
    "\n",
    "    U = csc_matrix(U, dtype=np.float32)\n",
    "    S = csc_matrix(S, dtype=np.float32)\n",
    "    Vt = csc_matrix(Vt, dtype=np.float32)\n",
    "    \n",
    "    return U, S, Vt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "U: [[-0.12560098 -0.07120612 -0.67528343  0.2126024 ]\n",
      " [-0.30083635 -0.22965695  0.13728355  0.26864198]\n",
      " [ 0.35730523  0.69205993  0.03599938  0.2890629 ]\n",
      " [ 0.10226665  0.36700046 -0.01981029  0.20552577]\n",
      " [ 0.12230982 -0.06416993 -0.0145381   0.17899378]\n",
      " [ 0.6340735  -0.21843855  0.08607852  0.25830176]\n",
      " [ 0.12816228 -0.4296043   0.3552903   0.51668966]\n",
      " [ 0.14948076 -0.17731705 -0.61420155  0.2696075 ]\n",
      " [-0.54755586  0.24640769  0.11118256  0.5642859 ]]\n",
      "S: [[1.7759248 0.        0.        0.       ]\n",
      " [0.        2.1746438 0.        0.       ]\n",
      " [0.        0.        2.4771335 0.       ]\n",
      " [0.        0.        0.        3.9029915]]\n",
      "Vt: [[ 0.02142367  0.02894757  0.12110947 -0.6533094   0.74647325]\n",
      " [-0.13377032  0.8824859  -0.2740653  -0.27630818 -0.2277413 ]\n",
      " [-0.9199122   0.04993301  0.13221717  0.27374363  0.24259274]\n",
      " [ 0.3450876   0.46640113  0.5286311   0.5181752   0.33974728]]\n"
     ]
    }
   ],
   "source": [
    "U, S, Vt = compute_svd(SongRate, 4)\n",
    "\n",
    "U_list = U.toarray()\n",
    "S_list = S.toarray()\n",
    "Vt_list = Vt.toarray()\n",
    "\n",
    "print('U:', U_list)\n",
    "print('S:', S_list)\n",
    "print('Vt:', Vt_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pred: [3.441083]\n"
     ]
    }
   ],
   "source": [
    "SongListAverage = []\n",
    "for i in range(len(SongRate)):\n",
    "    SongRateSum = 0\n",
    "    for j in range(len(SongRate[0])):\n",
    "        SongRateSum += SongRate[i][j]    \n",
    "    SongListAverage.append(SongRateSum / (len(SongRate[0])))\n",
    "    \n",
    "U_SongRate = U[0, :]\n",
    "S_SongRate = S[:, :]\n",
    "Vt_SongRate = Vt[:, 0]\n",
    "\n",
    "a = (U_SongRate * S_SongRate * Vt_SongRate).toarray()[0]\n",
    "#print('a:', a)\n",
    "\n",
    "pred = SongListAverage[0] + a\n",
    "\n",
    "print('pred:', pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
